import requests
import time
import datetime
from pyquery import PyQuery
import re
import pandas as pd
import json
from cases.helper import append_df_to_excel
# 考研调剂V
# ['法律硕士（法学）','法律硕士（非法学）']专硕
# ['军事法学','刑法学','国际法学','宪法学与行政法学','教育法学','民商法学','法学理论','法律史','经济法学','诉讼法学']学硕

# 考研调剂V，传入:法律硕士（法学）、专硕
def adjust(studyType, degreeType1):
    header = {
        'Content-Type': 'application/json',
        'Authorization': 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxNDQ3MSAxNjA2NjE1MjMxMjMyIn0.LwzQ7xefc_nPh6YAIn4Sw4E2W17E8mPyHNUQGMWv-Xo'
    }
    # with pd.ExcelWriter("C:\\Users\\86182\\Desktop\\择校信息爬取\\infoSchool.xlsx", mode='a', engine='openpyxl') as writer:
    writer = pd.ExcelWriter("C:\\Users\\86182\\Desktop\\择校信息爬取\\infoSchool.xlsx", mode='a', engine='openpyxl')
    dfTitle = pd.DataFrame(data=['信息名称', '学校省份', '专业', '专业门类', '学习方式', '发布时间', '补充说明', '学校名称', '学位类型'])
    dfTitle1 = dfTitle.transpose()
    dfTitle1.to_excel(excel_writer=writer, sheet_name=studyType, header=False, index=False, startrow=0, startcol=0)
    page = 1
    pageTotal = 100
    while page <= pageTotal:
        url = 'https://common-mini.okaoyan.com/api/adjust/info/list'
        body = {
            'province_name': '',
            'university_name': '',
            'major_name': studyType,
            'learn_style': '',
            'page_num': page,
            'page_size': 20,
            'year': 2020,
            'only_intention': 'false'
        }
        res = requests.get(url=url, params=body, headers=header)
        message = res.json()['message']
        try:
            assert message == '成功执行', '数据获取失败'
        except AssertionError as e:
            print(e)
            print('第{}页'.format(page))
            page += 1
            continue
        pageTotal = res.json()['data']['pageTotal']
        adjustInfoList = res.json()['data']['adjustInfoList']
        num = 1
        for j in range(len(adjustInfoList)):
            try:
                title = adjustInfoList[j]['title']  # 信息名称
            except Exception as e:
                title = ''
                print(e)
                print(adjustInfoList[j])
            try:
                provinceName = adjustInfoList[j]['provinceName']  # 学校省份
            except Exception as e:
                provinceName = ''
                print(e)
                print(adjustInfoList[j])
            try:
                majorName = adjustInfoList[j]['majorName']  # 专业
            except Exception as e:
                majorName = ''
                print(e)
                print(adjustInfoList[j])
            try:
                if adjustInfoList[j]['is985'] == 0 and adjustInfoList[j]['is211'] == 0:
                    majorCategory = ''  # 专业门类
                elif adjustInfoList[j]['is985'] == 1 and adjustInfoList[j]['is211'] == 0:
                    majorCategory = '985'
                elif adjustInfoList[j]['is985'] == 0 and adjustInfoList[j]['is211'] == 1:
                    majorCategory = '211'
                else:
                    majorCategory = '985/211'
            except Exception as e:
                majorCategory = ''
                print(e)
                print(adjustInfoList[j])
            try:
                learnStyle = adjustInfoList[j]['learnStyle']  # 学习方式
            except Exception as e:
                learnStyle = ''
                print(e)
                print(adjustInfoList[j])
            try:
                date = adjustInfoList[j]['publishDate']
                fmt = '%Y-%m-%d %H:%M:%S'
                time_tuple = time.strptime(date, fmt)
                year, month, day = time_tuple[:3]
                a_date = datetime.date(year, month, day)
                publishDate = a_date.strftime('%Y-%m-%d')  # 发布时间
            except Exception as e:
                publishDate = ''
                print(e)
                print(adjustInfoList[j])
            try:
                info = adjustInfoList[j]['content']
                if '<img' in info:
                    url = re.findall(r'([^-]src=\\?"(.+?)")', info)
                    urls = str(url)
                    contentText = PyQuery(info).text()
                    content = urls+contentText  # 补充说明（图片+文字）
                else:
                    content = PyQuery(info).text()  # 补充说明（文字）
            except Exception as e:
                content = ''
                print(e)
                print(adjustInfoList[j])
            try:
                universityName = adjustInfoList[j]['universityName']  # 学校名称
            except Exception as e:
                universityName = ''
                print(e)
                print(adjustInfoList[j])
            degreeType = degreeType1  # 学位类型
            df = pd.DataFrame(data=[title, provinceName, majorName, majorCategory, learnStyle, publishDate, content,
                                    universityName, degreeType])
            df1 = df.transpose()
            df1.to_excel(excel_writer=writer, sheet_name=studyType, header=False, index=False,
                         startrow=((page-1)*20 + num), startcol=0)
            num += 1
            writer.save()
        page += 1
        time.sleep(3)
        continue
    writer.close()


# 中公考研择校，需要数据的类别的id、信息等获取
# 专硕:{'classId': '', 'type': 2}
def getZSinfo():
    dataType = []  # 需要爬起数据的分类列表
    dataDict = {}  # 每个分类获取数据时，需要提交的参数
    header = {
        'Content-Type': 'application/json',
        'Authorization': '39_eRs7vfZyGuM-lVLYJEzgtvu3i9BkLZ9Pp8ApI45zTPwRsG-cA6R3tXctzl-TZnaJw7pUZbnSbRpEDbQIw4syi-Gue'
                         'A_nPYL8iMrulux3EZ9mG7CeTJAunIH8vCJ1GYNgjqdBjyXlzMpPMJAcBIHhACASZY'
    }
    url = 'https://kymp.kaoyan365.cn//major/getAllMajorsByCid'
    body = {'classId': '', 'type': 2}
    res = requests.post(url=url, data=json.dumps(body), headers=header)
    resCode = res.json()['code']
    assert resCode == 200, '专硕分类下，类别id等获取失败'
    legalList = res.json()['data']['法律']
    for i in legalList:
        dataDict['name'] = i['name']
        dataDict['code'] = i['code']
        if i['is_self'] == 2:
            dataDict['cid'] = ''
        elif i['is_self'] == 1:
            dataDict['cid'] = i['self_cid']
        dataType.append(dataDict.copy())
    return dataType

# 学硕:{'classId': 3, 'type': 1}
def getXSinfo():
    dataType = []  # 需要爬起数据的分类列表
    dataDict = {}  # 每个分类获取数据时，需要提交的参数
    header = {
        'Content-Type': 'application/json',
        'Authorization': '39_eRs7vfZyGuM-lVLYJEzgtvu3i9BkLZ9Pp8ApI45zTPwRsG-cA6R3tXctzl-TZnaJw7pUZbnSbRpEDbQIw4syi-Gue'
                         'A_nPYL8iMrulux3EZ9mG7CeTJAunIH8vCJ1GYNgjqdBjyXlzMpPMJAcBIHhACASZY'
    }
    url = 'https://kymp.kaoyan365.cn//major/getAllMajorsByCid'
    body = {'classId': 3, 'type': 1}
    res = requests.post(url=url, data=json.dumps(body), headers=header, timeout=120)
    resCode = res.json()['code']
    assert resCode == 200, '学硕分类下，类别id等获取失败'
    legalList = res.json()['data']['法学']
    for i in legalList:
        dataDict['name'] = i['name']
        dataDict['code'] = i['code']
        if i['is_self'] == 2:
            dataDict['cid'] = ''
        elif i['is_self'] == 1:
            dataDict['cid'] = i['self_cid']
        dataType.append(dataDict.copy())
    return dataType

# 中公考研择校数据爬取，传入学位类型：专硕还是学硕；和分类列表
def schoolSelection(dataType, degreeType):
    header = {
        'Content-Type': 'application/json',
        'Authorization': '39_eRs7vfZyGuM-lVLYJEzgtvu3i9BkLZ9Pp8ApI45zTPwRsG-cA6R3tXctzl-TZnaJw7pUZbnSbRpEDbQIw4syi-Gue'
                         'A_nPYL8iMrulux3EZ9mG7CeTJAunIH8vCJ1GYNgjqdBjyXlzMpPMJAcBIHhACASZY'
    }
    dfTitle = pd.DataFrame(data=['院校名称', '院校属性', '院校类型', '学科编号', '学院信息', '专业', '专业等级',
                                 '初试①年份', '初试①拟招人数', '初试①推免人数', '初试①总分数线', '初试①政治/管综分数线',
                                 '初试①外语分数线', '初试①业务课一分数线', '初试①业务课二分数线', '初试①政治/管综',
                                 '初试①外语', '初试①业务课一', '初试①业务课二', '初试①参考书目', '初试②年份', '初试②拟招人数',
                                 '初试②推免人数', '初试②总分数线', '初试②政治/管综分数线', '初试②外语分数线', '初试②业务课一分数线',
                                 '初试②业务课二分数线', '初试②政治/管综', '初试②外语', '初试②业务课一', '初试②业务课二',
                                 '初试②参考书目', '复试①年份', '复试①差额', '复试①科目', '复试①形式', '复试①口语',
                                 '复试①实操', '复试②年份', '复试②差额', '复试②科目', '复试②形式', '复试②口语',
                                 '复试②实操', '研究方向', '就业前景', '学位类型'])
    dfTitle1 = dfTitle.transpose()
    append_df_to_excel(filename="C:\\Users\\86182\\Desktop\\择校信息爬取\\中公考研.xlsx", df=dfTitle1, sheet_name='中公考研（学硕）',
                       header=False, index=False)
    # 院校属性写入
    propertyList = ['普通院校', '985', '211', '一流大学', '一流学科', '自划线', '推荐免试', '博士点']
    # 专业等级，返回的rank值-1作为索引
    rankList = ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C', 'C']
    # 院校类型，返回的yxtype值-1作为索引
    yxtypeList = ['综合类', '理工类', '艺术类', '体育类', '财经类', '政法类', '语言类', '医药类', '农林类', '军事类',
                  '民族类', '师范类']
    for dataInfo in dataType:
        page = 1
        totalPage = 100
        while page <= totalPage:
            urlBasicInfo = 'https://kymp.kaoyan365.cn//find/getCollegeMajor'
            bodyBasicInfo = {
                "codeList": dataInfo['code'],
                "cid": dataInfo['cid'],
                "propertyList": "",
                "yxtypeList": "",
                "provinceId": "",
                "rank": "",
                "isMath": "",
                "pageNo": page
            }
            resBasicInfo = requests.post(url=urlBasicInfo, data=json.dumps(bodyBasicInfo), headers=header, timeout=120)
            codeBasicInfo = resBasicInfo.json()['code']
            try:
                assert codeBasicInfo == 200
            except AssertionError as e:
                print(e)
                print('第{}页数据获取失败'.format(page))
                page += 1
                continue
            totalPage = resBasicInfo.json()['data']['pages']
            allList = resBasicInfo.json()['data']['list']
            for listInfo in allList:
                majorId = listInfo['majorId']
                try:
                    cname = listInfo['cname']  # 院校名称
                except Exception as e:  # 以后只捕获指定异常
                    cname = ''
                    print(e)
                    print(listInfo)
                try:
                    propertyInfo = []  # 院校属性
                    propertyStr = listInfo['property']
                    a = list(propertyStr.replace(',', ''))
                    c = [int(b) for b in a]
                    for d in c:
                        propertyOne = propertyList[d]
                        propertyInfo.append(propertyOne)
                except Exception as e:
                    propertyInfo = ''
                    print(e)
                    print(listInfo)
                try:
                    yxType = yxtypeList[int(listInfo['yxtype'])-1]  # 院校类型
                except Exception as e:
                    yxType = ''
                    print(e)
                    print(listInfo)
                try:
                    code = listInfo['code']  # 学科编号
                except Exception as e:
                    code = ''
                    print(e)
                    print(listInfo)
                try:
                    schoolName = listInfo['schoolName']  # 学院信息
                except Exception as e:
                    schoolName = ''
                    print(e)
                    print(listInfo)
                try:
                    majorName = listInfo['majorName']  # 专业
                except Exception as e:
                    majorName = ''
                    print(e)
                    print(listInfo)
                try:
                    rank = rankList[int(listInfo['rank'])-1]  # 专业等级
                except Exception as e:
                    rank = ''
                    print(e)
                    print(listInfo)
                # 初试信息获取
                urlFirstExam = 'https://kymp.kaoyan365.cn//result/getFirstExam'
                bodyFirstExam = {'majorId': majorId}
                resFirstExam = requests.get(url=urlFirstExam, params=bodyFirstExam, headers=header, timeout=120)
                codeFirstExam = resFirstExam.json()['code']
                assert codeFirstExam == 200, '初试信息获取失败，majorId为{}'.format(majorId)
                firstExamInfo = resFirstExam.json()['data']
                try:
                    yearOne = firstExamInfo[0]['year']  # 初试①年份
                except Exception as e:
                    yearOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    admission_numOne = firstExamInfo[0]['admission_num']  # 初试①拟招人数
                except Exception as e:
                    admission_numOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    tuimian_numOne = firstExamInfo[0]['tuimian_num']  # 初试①推免人数
                except Exception as e:
                    tuimian_numOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    total_markOne = firstExamInfo[0]['total_mark']  # 初试①总分数线
                except Exception as e:
                    total_markOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    politics_markOne = firstExamInfo[0]['politics_mark']  # 初试①政治/管综分数线
                except Exception as e:
                    politics_markOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    foreign_language_markOne = firstExamInfo[0]['foreign_language_mark']  # 初试①外语分数线
                except Exception as e:
                    foreign_language_markOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    buiness1_markOne = firstExamInfo[0]['buiness1_mark']  # 初试①业务课一分数线
                except Exception as e:
                    buiness1_markOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    buiness2_markOne = firstExamInfo[0]['buiness2_mark']  # 初试①业务课二分数线
                except Exception as e:
                    buiness2_markOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    politicsOne = firstExamInfo[0]['politics']  # 初试①政治/管综
                except Exception as e:
                    politicsOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    foreign_languageOne = firstExamInfo[0]['foreign_language']  # 初试①外语
                except Exception as e:
                    foreign_languageOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    buiness1One = firstExamInfo[0]['buiness1']  # 初试①业务课一
                except Exception as e:
                    buiness1One = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    buiness2One = firstExamInfo[0]['buiness2']  # 初试①业务课二
                except Exception as e:
                    buiness2One = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    refer_bookOne = firstExamInfo[0]['refer_book']  # 初试①参考书目
                except Exception as e:
                    refer_bookOne = ''
                    print(e)
                    print(firstExamInfo[0])
                try:
                    yearTwo = firstExamInfo[1]['year']  # 初试②年份
                except Exception as e:
                    yearTwo = ''
                    print(e)
                try:
                    admission_numTwo = firstExamInfo[1]['admission_num']  # 初试②拟招人数
                except Exception as e:
                    admission_numTwo = ''
                    print(e)
                try:
                    tuimian_numTwo = firstExamInfo[1]['tuimian_num']  # 初试②推免人数
                except Exception as e:
                    tuimian_numTwo = ''
                    print(e)
                try:
                    total_markTwo = firstExamInfo[1]['total_mark']  # 初试②总分数线
                except Exception as e:
                    total_markTwo = ''
                    print(e)
                try:
                    politics_markTwo = firstExamInfo[1]['politics_mark']  # 初试②政治/管综分数线
                except Exception as e:
                    politics_markTwo = ''
                    print(e)
                try:
                    foreign_language_markTwo = firstExamInfo[1]['foreign_language_mark']  # 初试②外语分数线
                except Exception as e:
                    foreign_language_markTwo = ''
                    print(e)
                try:
                    buiness1_markTwo = firstExamInfo[1]['buiness1_mark']  # 初试②业务课一分数线
                except Exception as e:
                    buiness1_markTwo = ''
                    print(e)
                try:
                    buiness2_markTwo = firstExamInfo[1]['buiness2_mark']  # 初试②业务课二分数线
                except Exception as e:
                    buiness2_markTwo = ''
                    print(e)
                try:
                    politicsTwo = firstExamInfo[1]['politics']  # 初试②政治/管综
                except Exception as e:
                    politicsTwo = ''
                    print(e)
                try:
                    foreign_languageTwo = firstExamInfo[1]['foreign_language']  # 初试②外语
                except Exception as e:
                    foreign_languageTwo = ''
                    print(e)
                try:
                    buiness1Two = firstExamInfo[1]['buiness1']  # 初试②业务课一
                except Exception as e:
                    buiness1Two = ''
                    print(e)
                try:
                    buiness2Two = firstExamInfo[1]['buiness2']  # 初试②业务课二
                except Exception as e:
                    buiness2Two = ''
                    print(e)
                try:
                    refer_bookTwo = firstExamInfo[1]['refer_book']  # 初试②参考书目
                except Exception as e:
                    refer_bookTwo = ''
                    print(e)
                # 复试信息获取
                urlReExam = 'https://kymp.kaoyan365.cn//result/getReExam'
                bodyReExam = {'majorId': majorId}
                resReExam = requests.get(url=urlReExam, params=bodyReExam, headers=header, timeout=120)
                codeReExam = resReExam.json()['code']
                assert codeReExam == 200, '复试信息获取失败，majorId为{}'.format(majorId)
                reExamInfo = resReExam.json()['data']
                try:
                    reYearOne = reExamInfo[0]['year']  # 复试①年份
                except Exception as e:
                    reYearOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reBalanceOne = reExamInfo[0]['balance']  # 复试①差额
                    if reBalanceOne == 0:
                        reBalanceOne = ''
                except Exception as e:
                    reBalanceOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reSubjectOne = reExamInfo[0]['subject']  # 复试①科目
                except Exception as e:
                    reSubjectOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reFushi_typeOne = reExamInfo[0]['fushi_type']  # 复试①形式
                except Exception as e:
                    reFushi_typeOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reIs_spokenOne = reExamInfo[0]['is_spoken']  # 复试①口语
                except Exception as e:
                    reIs_spokenOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reIs_handleOne = reExamInfo[0]['is_handle']  # 复试①实操
                except Exception as e:
                    reIs_handleOne = ''
                    print(e)
                    print(reExamInfo[0])
                try:
                    reYearTwo = reExamInfo[1]['year']  # 复试②年份
                except Exception as e:
                    reYearTwo = ''
                    print(e)
                try:
                    reBalanceTwo = reExamInfo[1]['balance']  # 复试②差额
                    if reBalanceTwo == 0:
                        reBalanceTwo = ''
                except Exception as e:
                    reBalanceTwo = ''
                    print(e)
                try:
                    reSubjectTwo = reExamInfo[1]['subject']  # 复试②科目
                except Exception as e:
                    reSubjectTwo = ''
                    print(e)
                try:
                    reFushi_typeTwo = reExamInfo[1]['fushi_type']  # 复试②形式
                except Exception as e:
                    reFushi_typeTwo = ''
                    print(e)
                try:
                    reIs_spokenTwo = reExamInfo[1]['is_spoken']  # 复试②口语
                except Exception as e:
                    reIs_spokenTwo = ''
                    print(e)
                try:
                    reIs_handleTwo = reExamInfo[1]['is_handle']  # 复试②实操
                except Exception as e:
                    reIs_handleTwo = ''
                    print(e)
                # 研究方向信息获取
                urlResearch = 'https://kymp.kaoyan365.cn//result/getResearchInfo'
                bodyResearch = {'majorId': majorId}
                resResearch = requests.get(url=urlResearch, params=bodyResearch, headers=header, timeout=120)
                codeResearch = resResearch.json()['code']
                assert codeResearch == 200, '研究方向信息获取失败，majorId为{}'.format(majorId)
                try:
                    allSearchInfo = resResearch.json()['data']
                    researchDction = []  # 研究方向
                    for searchInfo in allSearchInfo:
                        research_name = searchInfo['research_name']
                        researchDction.append(research_name)
                except Exception as e:
                    researchDction = ''
                    print(e)
                    print(resResearch)
                # 就业前景信息获取
                urlMajob = 'https://kymp.kaoyan365.cn//result/queryMajorById'
                bodyMajob = {'majorId': majorId}
                resMajob = requests.get(url=urlMajob, params=bodyMajob, headers=header, timeout=120)
                codeMajob = resMajob.json()['code']
                assert codeMajob == 200, '就业前景信息获取失败，majorId为{}'.format(majorId)
                try:
                    prospect = resMajob.json()['data']['prospect']  # 就业前景
                except Exception as e:
                    prospect = ''
                    print(e)
                    print(resMajob)
                df = pd.DataFrame(data=[cname, propertyInfo, yxType, code, schoolName, majorName, rank, yearOne,
                                        admission_numOne, tuimian_numOne, total_markOne, politics_markOne,
                                        foreign_language_markOne, buiness1_markOne, buiness2_markOne, politicsOne,
                                        foreign_languageOne, buiness1One, buiness2One, refer_bookOne, yearTwo,
                                        admission_numTwo, tuimian_numTwo, total_markTwo, politics_markTwo,
                                        foreign_language_markTwo, buiness1_markTwo, buiness2_markTwo, politicsTwo,
                                        foreign_languageTwo, buiness1Two, buiness2Two, refer_bookTwo, reYearOne,
                                        reBalanceOne, reSubjectOne, reFushi_typeOne, reIs_spokenOne, reIs_handleOne,
                                        reYearTwo, reBalanceTwo, reSubjectTwo, reFushi_typeTwo, reIs_spokenTwo,
                                        reIs_handleTwo, researchDction, prospect, degreeType])
                df1 = df.transpose()
                append_df_to_excel(filename="C:\\Users\\86182\\Desktop\\择校信息爬取\\中公考研.xlsx", df=df1,
                                   sheet_name='中公考研（学硕）', header=False, index=False)
            page += 1
            time.sleep(1)



if __name__ == '__main__':
    allInfo = getXSinfo()
    header = {
        'Content-Type': 'application/json',
        'Authorization': '39_eRs7vfZyGuM-lVLYJEzgtvu3i9BkLZ9Pp8ApI45zTPwRsG-cA6R3tXctzl-TZnaJw7pUZbnSbRpEDbQIw4syi-Gue'
                         'A_nPYL8iMrulux3EZ9mG7CeTJAunIH8vCJ1GYNgjqdBjyXlzMpPMJAcBIHhACASZY'
    }
    totalNum = 0
    for dataInfo in allInfo:
        urlBasicInfo = 'https://kymp.kaoyan365.cn//find/getCollegeMajor'
        bodyBasicInfo = {
            "codeList": dataInfo['code'],
            "cid": dataInfo['cid'],
            "propertyList": "",
            "yxtypeList": "",
            "provinceId": "",
            "rank": "",
            "isMath": "",
            "pageNo": 1
        }
        resBasicInfo = requests.post(url=urlBasicInfo, data=json.dumps(bodyBasicInfo), headers=header, timeout=120)
        codeBasicInfo = resBasicInfo.json()['code']
        assert codeBasicInfo == 200, '获取数据失败'
        numEve = resBasicInfo.json()['data']['total']
        totalNum += numEve
    print(totalNum)
